package com.tl.spark.scala

import java.io.File

import org.apache.spark.{SparkConf, SparkContext}


/**
  * @program: spark-test
  * @description:
  * @author: dong.tl
  * @create: 2018-09-12 17:50
  **/
object ScalaReadFile {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("ScalaWordCount").setMaster("local[2]")

    val sc = new SparkContext(conf)

//    val tuples = sc.binaryFiles("F:\\IdeaProjects\\新建文件夹\\spark-test").collect()

  val input = "F:\\IdeaProjects\\新建文件夹\\spark-test\\"

//    sc.stop()
  val files = sc.binaryFiles(input,12)
//  sc.wholeTextFiles(input).collect()
    sc.listFiles().foreach(file=>{
      println(file)
    })
    files.foreach(file =>{
      val f = new File(file._1)
      println(f.getName)
    })
    println(sc.files.size)

    sc.stop()

  }

  def subdirs3(dir: File): Iterator[File] = {
    val d = dir.listFiles.filter(_.isDirectory)
    val f = dir.listFiles.toIterator
    f ++ d.toIterator.flatMap(subdirs3 _)
  }
}
